#include "common.h"
#include <boost/filesystem.hpp>


#include "util.hpp"


struct DocInfo{
    std::string title;
    std::string content;
    std::string url;
};

std::string src_path = "./data/input";
std::string out_path = "./data/output/raw.txt";

bool enumHtmls(const std::string &src_path ,std::vector<std::string> *html_files);
bool parseHtmls(const std::vector<std::string> &html_files, std::vector<DocInfo> *docs);
bool saveHtmls(const std::vector<DocInfo> &docs, const std::string &output_file);

int main(){
    std::vector<std::string> html_files;
    if(!enumHtmls(src_path ,&html_files)){
        std::cerr << "Enum File Error\n";
        return 1;
    }
    std::vector<DocInfo> docs;
    if(!parseHtmls(html_files,&docs)){
        std::cerr << "Parse Htmls Error\n";
        return 2;
    }

    if(!saveHtmls(docs, out_path)){
        std::cerr << "Generate The Raw Document txt Failed\n";
        return 3;
    }

    return 0;
}
bool enumHtmls(const std::string &src_path ,std::vector<std::string> *html_files){
    namespace fs = boost::filesystem;
    fs::path src = fs::path(src_path);
    if(!fs::exists(src)) return false;
    // directory.hpp:890 -> recursive_directory_iterator() BOOST_NOEXCEPT {} // creates the "end" iterator
    fs::recursive_directory_iterator end;
    // 递归遍历src_path
    for(fs::recursive_directory_iterator file_iter(src); file_iter != end; file_iter++){
        if(!file_iter->is_regular_file()) continue;
        if(!(file_iter->path().extension().string() == ".html")) continue;
        std::string html_path = file_iter->path().string();
        // std::cout << html_path + "\n";
        html_files->push_back(html_path);
    }
    return true;
}
bool getTitle(const std::string &html_content, std::string *out){
    size_t begin = html_content.find("<title>");
    if(begin == std::string::npos) return false;
    size_t end = html_content.find("</title>");
    if(end == std::string::npos) return false;
    
    begin += std::string("<title>").size();
    if(begin > end) return false;
    *out = html_content.substr(begin, end - begin);
    return true;
}
bool getContent(const std::string &html_content, std::string *out){
    enum class State : uint8_t {
        Lable,
        Content
    };
    State s = State::Lable;
    for(char c : html_content){
        switch(s){
            case State::Lable :
                if(c == '>') s = State::Content;
                break;
            case State::Content :
                if(c == '<') s = State::Lable;
                else{
                    if(c == '\n') c = ' ';
                    out->push_back(c);
                }
                break;
        }
    }
    return true;
}
bool getUrl(const std::string &file_path, std::string *out){
    std::string url_head = "https://www.boost.org/doc/libs/1_78_0/doc/html";
    std::string url_tail = file_path.substr(src_path.size());
    *out = url_head + url_tail;
    return true;
}

//debug 
static void showDoc(const DocInfo &doc){
    std::cout << "title: " << doc.title << "\n";
    std::cout << "content: " << doc.content << "\n";
    std::cout << "url: " << doc.url << "\n";
}
bool parseHtmls(const std::vector<std::string> &html_files, std::vector<DocInfo> *docs){
    for(const std::string &file : html_files){
        std::string html_content;
        DocInfo doc;
        if(!util_ns::FileUtil::readFile(file, &html_content)){
            return false;
        }
        if(!getTitle(html_content,&doc.title)){
            return false;
        }
        if(!getContent(html_content, &doc.content)){
            return false;
        }
        if(!getUrl(file, &doc.url)){
            return false;
        }
        docs->push_back(std::move(doc));
        // sleep(1);
        // showDoc(doc);
    }
    return true;
}
bool saveHtmls(const std::vector<DocInfo> &docs, const std::string &output_file){
    std::ofstream out(output_file, std::ios::out | std::ios::binary);
    if(!out.is_open()){
        std::cerr << "Open file " << output_file << " failed\n";
        return false;
    }
    std::string joint_string;
    for(const DocInfo &doc : docs){
        joint_string = doc.title;
        joint_string += separator;
        joint_string += doc.content;
        joint_string += separator;
        joint_string += doc.url;
        joint_string += "\n";
        out.write(joint_string.c_str(), joint_string.size());
    }
    out.close();
    return true;
}